#/bin/bash
#ner automation

#source /home/yzfu/nlp/myproject/venv/tensorflow/bin/activate
#ner模型训练部分 python2.7


stage=7
mark=mark.txt
data_root_dir=/home/yzfu/nlp/kg_abc/fyz_kg_nlp/abc_project_data/
origin_Data_path=text_01/
clear_data_path=text_02/
origin_marked_html_path=html_01/
word_data=text_03/
word_seg_dict=new_dict.txt
stop_word_dict=stopword.dict
word_train_data=abc.train
word_test_data=abc.dev
flag_add_mark=flase
analysed_flag=flase
html_dataset_path=tmp_article/
result_dataset_path=ner_html_result_11.23/
tags_file=tags.txt

#process of traning ner model
# （1）train/01_extract_mark_data.py	
if [ $stage -le 1 ]; then
	echo "step1:get original marked text and tags dict"
	cd $data_root_dir
	#if [ -d "$origin_Data_path" ];then
	#	rm -rf ${origin_Data_path}
	#	mkdir $origin_Data_path
	#fi
	#if [ -d "$origin_marked_html_path" ];then
	#	rm -rf ${origin_marked_html_path}
	# 	mkdir $origin_marked_html_path
	#fi
	if [ -d "$clear_data_path" ];then
		rm -rf ${clear_data_path}
		mkdir $clear_data_path
	fi
	cd -
 	python ./train/01_extract_mark_data.py --home_dir_ $data_root_dir \
		--text_Path_ $origin_Data_path \
		--htmlPath_ $origin_marked_html_path \
		--mark_path_ $mark \
		--dict_path_ dict.txt \
		--dest_path_ $word_seg_dict \
		--data_clear $clear_data_path ||exit 1;
	echo "complete......"
fi
#添加自定义词典(添加到mark.txt和dict.txt)
if [ "$flag_add_mark" == "true" ]; then
	dict_dir="xxxx"
	python add_dict.py
fi

# （2）train/02_prepare_lstm_train_tag.py mark标注替换词性
# new_dict.txt 通过ner目录下的generate_marked_data.py生成sql_dict.txt，然后命名成new_dict.txt即可
if [ $stage -le 2 ]; then
	echo "step2:Word Segmentation and Data Annotations"
	cd $data_root_dir
	if [ -d "$word_data" ];then
		rm -rf ${word_data}
		mkdir $word_data
	fi
	cd -
	python ./train/02_fyz_ner_data.py --home_dir_ $data_root_dir \
		--text_Path_ $clear_data_path \
		--mark_path_ $mark \
		--product_03_ $word_data \
		--wordseg_dict $word_seg_dict \
		--stopword $stop_word_dict \
		--extract_dict true ||exit 1;
	echo "complete......"
fi

# （3）train/03_genera_lstm_tag_data.py 按照文档划分训练测试数据
if [ $stage -le 3 ]; then
	echo "step3:devide the data into training dataset and development dataset"
	cd $data_root_dir
	if [ -f "$word_train_data" ];then
		rm -rf $word_train_data
	fi
	if [ -f "$word_test_data" ];then
		rm -rf $word_test_data
	fi
	cd -
	python ./train/03_genera_lstm_tag_data.py --home_dir_ $data_root_dir \
		--train_path_ $word_train_data \
		--dev_path_ $word_test_data \
		--product_data_dir_ $word_data ||exit 1;
	echo "complete ......"
fi

# （4）ner/data/product_abc.train.py 将（3）中的训练数据合成句子
#if [ $stage -le 4 ]; then
#	echo "step4:combine the word_data and build up sentence"
#	cd $data_root_dir
#	if [ -f "${word_train_data}_fyz" ];then
#		rm -rf ${word_train_data}_fyz
#	fi
#	if [ -f "${word_test_data}_fyz" ];then
#		rm -rf ${word_test_data}_fyz
#	fi
#	cd -
#	python ./ner/data/product_abc.train.py --home_dir_ $data_root_dir \
#		--input_flie_ ${word_train_data} \
#		--output_flie_ ${word_train_data}_fyz ||exit 1;
#	python ./ner/data/product_abc.train.py --home_dir_ $data_root_dir \
#		--input_flie_ ${word_test_data} \
#		--output_flie_ ${word_test_data}_fyz ||exit 1;
#	echo "complete ......"
#fi
#build_data.py 根据训练模型生成词典（源代码自带脚本）

if [ $stage -le 5 ]; then
	echo "step5:build up dataset......"
	python ./ner/build_data.py ||exit1;
	echo "complete......"
fi
export CUDA_VISIBLE_DEVICES=0
#开始训练模型
if [ $stage -le 6 ]; then
	echo "step6:training model......"
	python ./ner/train.py ||exit1;
	echo "complete......"
fi

########################批量识别（解码）部分############################
#开始获取未标注的html文档（未解析）该部分代码在python3环境下运行，由于非
#本人编写，建议在pycharm上运行，否侧得修改outline/gol.cfg部分的路径 
#if [ $stage -le 6 ]; then
#	echo "step6:begin to get html text"
#	python3.5 ./outline/test.py ||exit1;
#	echo "complete......"
#fi
##############################################################
#多线程批量识别

export CUDA_VISIBLE_DEVICES=0
if [ $stage -le 7 ]; then
	echo "step7:begin to ner"
	python ./ner/new_tread_ner.py --home_dir_ $data_root_dir \
		--html_dataset $html_dataset_path \
		--result_dataset $result_dataset_path \
		--tags_path $tags_file ||exit 1;
	echo "complete......"
fi
<<mark
mark
